package com.xiaotu.spider.project.pageprocessor;

import com.xiaotu.common.exception.SpiderException;
import com.xiaotu.common.exception.SpiderExceptionCode;
import com.xiaotu.common.util.*;
import com.xiaotu.spider.APageProcessor;
import com.xiaotu.spider.downloader.SpiderHttpClientDownloader;
import net.sf.json.JSONObject;
import org.apache.commons.lang3.StringUtils;
import org.springframework.context.annotation.Scope;
import org.springframework.stereotype.Component;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Request;
import us.codecraft.webmagic.selector.Html;

import java.util.*;

/**
 * @类名 WeiboFansPageProcessor
 * @日期 2017年8月29日
 * @作者 高海军
 * @功能 微博话题粉丝数据爬取
 */
@Component("WeiboTopicFansPageProcessor")
@Scope("prototype")
public class WeiboTopicFansPageProcessor extends APageProcessor
{
    private static final String SEARCH_URL = "http://huati.weibo.com/k/";

    private static final String ESCAPE_TOPICID = "#topicId#";
    private static final String ESCAPE_PAGENO = "#pageNo#";
    private static final String ESCAPE_LASTPAGENO = "#lastPageNo#";

    private static final String FANS_LIST_URL = "http://weibo.com/p/"
            + ESCAPE_TOPICID
            + "/followlist?pids=Pl_Core_F4RightUserList__39&page="
            + ESCAPE_PAGENO + "&ajaxpagelet=1&ajaxpagelet_v6=1&__ref=/p/"
            + ESCAPE_TOPICID + "/followlist?page=" + ESCAPE_LASTPAGENO
            + "#Pl_Core_F4RightUserList__39&_t=FM_";

    private static final String TOPIC_URL_STAR = "http://weibo.com/p/";
    private static final String TOPIC_URL_END = "/followlist";

    private static final String SCRIPT_PARENT_START = "<script>parent.FM.view(";
    private static final String SCRIPT_FM_START = "<script>FM.view(";
    private static final String SCRIPT_END = ")</script>";

    private static final String LABELS_LABEL = "标签信息";
    private static final String LABELS_EDU = "教育信息";
    private static final String LABELS_JOB = "工作信息";
    private static final String LABELS_BASIC = "基本信息";

    private static final List<String> LABELS_ALL = Arrays.asList("所在地", "性别",
            "生日", "简介", LABELS_LABEL, LABELS_EDU, LABELS_JOB);

    private String fileName;
    private String inputFile;
    private int maxPageNo = 100;

    @Override
    public void initRunParams(Map<String, Object> map)
    {
        if (map.get("fileName") == null || map.get("inputFile") == null)
            throw new SpiderException(SpiderExceptionCode.PageProcessorParameterErrorException);
        fileName = FileUtil.getCacheFilePath(map.get("fileName") + StringUtils.EMPTY);
        inputFile = FileUtil.getCacheFilePath(map.get("inputFile") + StringUtils.EMPTY);
        if (map.get("pageNo") != null)
            maxPageNo = Integer.parseInt(map.get("pageNo") + StringUtils.EMPTY);
    }

    @Override
    protected void setSiteRequest()
    {
        List<String> tvList = FileUtil.readFileByLine(inputFile);
        if (tvList == null || tvList.isEmpty())
            throw new SpiderException(SpiderExceptionCode.PageProcessorParameterErrorException);
        List<JSONObject> objList = new ArrayList<>();
        tvList.forEach(line ->
        {
            JSONObject target = new JSONObject();
            target.put("tvName", line.trim());
            objList.add(target);
        });

        this.targetList = objList;
        int i = targetList.size();
        for (JSONObject target : targetList)
            this.addStartRequest(this.getStartRequest(target), target, i--);
    }

    @Override
    protected Request getStartRequest(JSONObject target)
    {
        target.putAll(this.getOtherTarget());
        return new Request(SEARCH_URL + KeywordEncode.encode(target.getString("tvName")));
    }

    private void topicProcess(Page page)
    {
        Map<String, Object> map = new HashMap<>();
        map.putAll(page.getRequest().getExtras());
        map.put(ESCAPE_TOPICID, this.getTopicId(page.getRawText()));
        map.put(ESCAPE_PAGENO, 1);
        PageUtils.addTargetRequest(page, this.getUrl(JSONObject.fromObject(map)), map);
    }

    /**
     * 获取粉丝列表的url地址
     *
     * @param target 话题地址
     * @return url
     */
    private String getUrl(JSONObject target)
    {
        int pageNo = target.getInt(ESCAPE_PAGENO);
        String pageNoSuf = StringUtils.EMPTY;
        if (pageNo > 2)
            pageNoSuf = "?page=" + (pageNo - 1);
        target.put(SpiderHttpClientDownloader.HEADER_REFERER,
                "http://weibo.com/p/" + target.getString(ESCAPE_TOPICID)
                        + "/followlist" + pageNoSuf);
        String url = FANS_LIST_URL
                .replaceAll(ESCAPE_TOPICID, target.getString(ESCAPE_TOPICID))
                .replaceAll(ESCAPE_PAGENO, pageNo + StringUtils.EMPTY)
                .replaceAll(ESCAPE_LASTPAGENO, (pageNo - 1) + StringUtils.EMPTY)
                + System.currentTimeMillis() + StringUtils.EMPTY
                + StringUtils.leftPad(
                MathUtil.getRand(1, 99) + StringUtils.EMPTY, 2, "0");
        String ref = "__ref=";
        int start = url.indexOf(ref) + ref.length();
        int end = start + url.substring(start).indexOf("&");
        url = url.substring(0, start)
                + KeywordEncode.encode(url.substring(start, end))
                + url.substring(end);
        return url;
    }

    /**
     * 从url中解析出话题id
     *
     * @param response html内容
     * @return 话题id
     */
    private String getTopicId(String response)
    {
        String data = RegexUtils.regexSearch("\\$CONFIG\\['page_id'\\]='(.*)';", response);
        if (StringUtils.isEmpty(data))
            throw new SpiderException(SpiderExceptionCode.DataInvalideException);
        data = RegexUtils.regexSearch("='(.*)'", data);
        data = data.replaceAll("'|=", StringUtils.EMPTY);
        return data;
    }

    @Override
    public void process(Page page)
    {
        if (page.getUrl().toString().contains(SEARCH_URL))
            this.topicProcess(page);
        else if (page.getUrl().toString().contains(TOPIC_URL_END))
            this.fansListProcess(page);
        else
            this.fansProcess(page);
    }

    /**
     * 基本信息解析
     *
     * @param html 页面元素
     * @param map  缓存
     */
    private void parseBaseinfo(Html html, Map<String, String> map)
    {
        List<String> liList = html.xpath("//li[@class='li_1 clearfix']").all();
        if (liList == null || liList.isEmpty())
            throw new SpiderException(
                    SpiderExceptionCode.DataInvalideException);
        for (String liStr : liList)
        {
            Html liObj = new Html(liStr);
            String label = liObj
                    .xpath("//span[@class='pt_title S_txt2']/text()").get();
            label = label.replace(SepratorUtil.SEP_COLON_CN, StringUtils.EMPTY);
            if (LABELS_ALL.contains(label))
                map.put(label,
                        liObj.xpath("//span[@class='pt_detail']/text()").get());
        }
    }

    /**
     * 标签信息解析
     *
     * @param html 页面元素
     * @param map  缓存
     */
    private void parseLabel(Html html, Map<String, String> map)
    {
        List<String> aList = html.xpath("//a/text()").all();
        if (aList == null || aList.isEmpty())
            return;
        StringBuffer buffer = new StringBuffer();
        for (String a : aList)
            buffer.append(SepratorUtil.SEP_COMMA_EN + a.trim());
        map.put(LABELS_LABEL,
                buffer.substring(SepratorUtil.SEP_COMMA_EN.length()));
    }

    /**
     * 其他信息解析
     *
     * @param html  页面元素
     * @param map   缓存
     * @param label 要解析的信息名称
     */
    private void parseHtml(Html html, Map<String, String> map, String label)
    {
        map.put(label, HtmlUtil
                .delHTMLTag(html.xpath("//span[@class='pt_detail']").get()));
    }

    /**
     * 即系粉丝的详细信息
     *
     * @param html 页面元素
     */
    private void parseInfo(Page page, Html html)
    {
        List<String> divList = html.xpath("//div[@class='WB_cardwrap S_bg2']")
                .all();
        if (divList == null || divList.isEmpty())
            throw new SpiderException(
                    SpiderExceptionCode.DataInvalideException);
        Map<String, String> map = new HashMap<String, String>();
        boolean ok = false;
        for (String divStr : divList)
        {
            Html divObj = new Html(divStr);
            String label = divObj.xpath("//div[@class='obj_name']/h2/text()")
                    .get();
            switch (label)
            {
                case LABELS_BASIC:
                    this.parseBaseinfo(divObj, map);
                    ok = true;
                    break;
                case LABELS_LABEL:
                    this.parseLabel(divObj, map);
                    break;
                case LABELS_EDU:
                case LABELS_JOB:
                    this.parseHtml(divObj, map, label);
                    break;
            }
        }

        if (!ok)
            throw new SpiderException(SpiderExceptionCode.DataInvalideException);

        StringBuffer buffer = new StringBuffer(StringUtil.nullToString(page.getRequest().getExtra("tvName")));
        buffer.append(SepratorUtil.SEP_CSV_FIELDS + StringUtil.nullToString(page.getRequest().getExtra("userId")));
        for (String key : LABELS_ALL)
            buffer.append(SepratorUtil.SEP_CSV_FIELDS + (map.containsKey(key) ? map.get(key) : StringUtils.EMPTY));
        System.out.println(buffer);
        FileUtil.writeFile(fileName, buffer.toString() + SepratorUtil.LINE_SEPRATOR);
    }

    /**
     * 粉丝详情处理
     *
     * @param page 页面对象
     */
    private void fansProcess(Page page)
    {
        int idx = 1;
        String response = null;
        while (true)
        {
            int start = StringUtils.lastOrdinalIndexOf(page.getRawText(),
                    SCRIPT_FM_START, idx);
            if (start < 0)
                break;
            int end = StringUtils.lastOrdinalIndexOf(page.getRawText(),
                    SCRIPT_END, idx);

            String temp = StringUtils.substring(page.getRawText(),
                    start + SCRIPT_FM_START.length(), end);
            if (temp.contains("基本信息"))
            {
                response = temp;
                break;
            }
            idx++;
        }
        if (StringUtils.isEmpty(response))
            throw new SpiderException(
                    SpiderExceptionCode.DataInvalideException);
        JSONObject jsonObj = JSONObject.fromObject(response);
        if (jsonObj == null || jsonObj.isEmpty() || !jsonObj.has("html"))
            throw new SpiderException(
                    SpiderExceptionCode.DataInvalideException);
        Html divObj = new Html(jsonObj.getString("html"));
        this.parseInfo(page, divObj);
    }

    /**
     * 粉丝列表处理：添加粉丝详情页面请求
     *
     * @param page 页面对象
     */
    @SuppressWarnings("unchecked")
    private void fansListProcess(Page page)
    {
        String response = page.getRawText();
        if (!response.startsWith(SCRIPT_PARENT_START))
            throw new SpiderException(
                    SpiderExceptionCode.DataInvalideException);

        response = response.replace(SCRIPT_PARENT_START, StringUtils.EMPTY)
                .replace(SCRIPT_END, StringUtils.EMPTY);
        JSONObject jsonObj = JSONObject.fromObject(response);
        if (jsonObj == null || jsonObj.isEmpty() || !jsonObj.has("html"))
            throw new SpiderException(
                    SpiderExceptionCode.DataInvalideException);
        Html html = new Html(jsonObj.getString("html"));
        List<String> liList = html.xpath("//li[@class='follow_item S_line2']")
                .all();
        for (String divStr : liList)
        {
            String userId = new Html(divStr)
                    .xpath("//a[@class='S_txt1']/strong/@usercard").get();
            userId = RegexUtils.regexSearch("^id=" + RegexUtils.REGEX_INTEGER,
                    userId);
            if (StringUtils.isEmpty(userId))
                continue;
            userId = userId.replace("id=", StringUtils.EMPTY);
            page.getRequest().putExtra("userId", userId);
            PageUtils.addTargetRequest(page,
                    "http://weibo.com/p/100505" + userId
                            + "/info?mod=pedit_more",
                    page.getRequest().getExtras(), true);// 粉丝详情页面
        }

        Map<String, Object> map = page.getRequest().getExtras();
        if (maxPageNo > (int) map.get(ESCAPE_PAGENO) && liList.size() > 0)
        {
            map.put(ESCAPE_PAGENO, (int) map.get(ESCAPE_PAGENO) + 1);
            JSONObject target = JSONObject.fromObject(map);
            PageUtils.addTargetRequest(page, this.getUrl(target),
                    new HashMap<String, Object>(target));// 粉丝列表下一页
        }

    }

    @Override
    protected String getSpiderType()
    {
        return "WeiboTopicFans";
    }

    @Override
    public String getPageDataType(Page page)
    {
        return "WeiboTopicFans";
    }
}
